const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

async function main() {
  let sub_tab_english = 'alibaba';
  let sub_tab = '阿里巴巴';
  const fileName = `infoq_${sub_tab_english}.json`;
  let totalBlogs = [];
  let tab = 'infoq';

  const browser = await puppeteer.launch({headless: false, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});
  page.on('console', msg => {
    console.log(msg.text());
  });

  page.on('pageerror', msg => {
    console.log(msg);
  });

  let url = `https://www.infoq.cn/topic/${sub_tab_english}`;
  await page.goto(url, {waitUntil: 'domcontentloaded'});
  await page.waitFor(1000);

  let preScrollHeight = 0;
  let scrollHeight = -1;
  let cnt = 0;
  let retry_cnt = 0;
  const SLEEP_TIME = 100;
  while (cnt < 150) {
    // 详情信息是根据滚动异步加载，所以需要让页面滚动到屏幕最下方，通过延迟等待的方式进行多次滚动
    let scrollH1 = await page.evaluate(async () => {
      let h1 = document.body.scrollHeight;
      window.scrollTo(0, h1);
      return h1;
    });
    // 延迟等待时间
    await page.waitFor(SLEEP_TIME);
    let scrollH2 = await page.evaluate(async () => {
      return document.body.scrollHeight;
    });
    let scrollResult = [scrollH1, scrollH2];
    preScrollHeight = scrollResult[0];
    scrollHeight = scrollResult[1];

    if (preScrollHeight == scrollHeight) {
      retry_cnt++;
      if (retry_cnt > 20) {
        break;
      }
      try {
        await page.click('#layout > div.inner-content > div.page-topic-list > div:nth-child(3) > div > div.com-more-button.loading-btn > div');
        cnt++;
        retry_cnt = 0;
      } catch (e) {
        // 容错处理
        await page.evaluate(x => {
          window.scrollTo(0, x - 2000)
        }, scrollHeight);
        await page.waitFor(1000);
        await page.evaluate(x => {
          window.scrollTo(0, x)
        }, scrollHeight);
      }
    } else {
      console.log('..............');
      cnt++;
    }
    console.log(scrollResult, cnt);
  }

  totalBlogs = await collectBlogs(page, tab, sub_tab);

  console.log(totalBlogs.length);

  let writerStream = fs.createWriteStream(`crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');

};


async function collectBlogs(page, tab, subTab) {
  let blogs = await page.$$eval('#layout > div.inner-content > div.page-topic-list > div:nth-child(3) > div > div.list > div', async (eles, tab, subTab) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('div.info > h6 > a').innerText.trim();
      let content = ele.querySelector('div.info > h6 > a').href;
      let source = ele.querySelector('div.info > p.editor.with-image > a').innerText.trim();
      tmpBlogs.push({
        title: title,
        content: content,
        tab: tab,
        sub_tab: subTab,
        source: source
      });
    });
    return tmpBlogs;
  }, tab, subTab);
  return blogs;
}

main();


